import datetime, sys, json
import pandas as pd
from dateutil.relativedelta import relativedelta
from src.base_class import BaseClass
from src.exts import TestLogger
from src.mxtf_api import MxtfApi
import pytz
import numpy as np
class shared_data():
    
    def __init__(self):
        pass
east8 = pytz.timezone('Asia/Shanghai')

# 计算器对象，包含损失计算对象，以及数据预处理等方法
class LossCalculator:
    def __init__(self,cltrdhist_json,test_mode=False,logger=None):
        self.logger = logger
        if not self.logger:
            self.logger = TestLogger()
        # 是否测试模式
        self.test_mode = test_mode
        # 数据库交互api
        self.mxtfapi = MxtfApi()
        if test_mode:
            self.TaskInfo ={  
                "caculateId":908494624421580801,
                "stock_no":"股票代码",
                "broker_id":"券商编号",
                "begin_time":11111,
                "end_time":11111,
                "trade_list":json.loads(cltrdhist_json)
                        }
        else:
            self.TaskInfo = cltrdhist_json
        # 预处理客户交易历史记录
        self.cl_trd_hist = self.prepare_cl_trd_hist(cltrdhist_json)
        # 提取客户交易历史记录中的所有个股代码
        self.stock_list = list(self.cl_trd_hist['stock_code'].unique())

        # self.stock_measure = None # 个股监管措施
        self.xujiachenshu_action = None # 虚假陈述行为
        self.stock_hist = None # 个股行情历史
        self.stock_trd_hist = None  # 个股交易历史
        self.mkt_index_hist = None # 大盘指数行情历史
        self.sw_ind_comp = None # 申万行业成分股
        self.sw_ind_comp_change_hist = None # 申万行业成分股变动历史
        self.sw_ind_index_hist = None # 申万行业指数行情历史
        
        # 股票代码转交易所标识字典
        self.stockcode2mkt = {'600':'sh','601':'sh','603':'sh','605':'sh','688':'sh','900':'sh',
                              '000':'sz','001':'sz','002':'sz','003':'sz','004':'sz','200':'sz','300':'sz'}
        # 交易所代码转大盘指数字典
        self.mkt2mktindex = {'sh':'sh000001','sz':'sz399001'}
        
        # 输出的关键属性列表
        self.key_attr = ['calculate_id','stock_code','commit_date','measure_date','correct_date','reveal_date','invest_valid',\
                         'invest_init_date','init_stock_quant','invest_clear_date','bench_mark_date','bench_mark_price',\
                         'cal_sys_loss_st_date','cal_sys_loss_ed_date','measure_type','measure_subtype','action_id','qinquan_level',\
                         'measure_loss','measure_sys_loss','measure_non_sys_loss','bench_mark_rt_list','result_remark',\
                         'real_loss','other_loss','net_loss']

        self.summary = {'total_loss':0, # 总的损失金额
                        'details':{}, # 每只个股的损失计算明细，只有损失最大的那次虚假陈述（监管发函）
                        'more_details':{} # 每只个股每次虚假陈述（监管发函）的损失计算明细
                        }

    # 定义一个损失计算对象，对每只个股的每次虚假陈述进行损失计算
    class measure_loss_calculator(BaseClass): 
        def __init__(self,outer_self,eachmeasure,logger=None):
            super().__init__(logger)
            self.slc = outer_self
            self.calculate_id = self.slc.calculate_id
            self.stock_code = eachmeasure['stockCode'] # 股票代码，直接从监管发函记录取，跟outer_self里面的理论上是一致的
            self.commit_date = eachmeasure['xujiachenshuDate'] # 实施日，发生虚假陈述的日期
            self.measure_date = eachmeasure['measureDate'] # 监管发函的日期
            self.correct_date = eachmeasure['gengzhengDate'] # 更正日
            self.measure_type = '' # 监管措施大类，暂时不会用到
            self.measure_subtype = '' # 监管措施类型，暂时不会用到
            # self.measure_type = eachmeasure['measureType'] # 监管措施大类
            # self.measure_subtype = eachmeasure['measureSubtype'] # 监管措施类型
            self.action_id = eachmeasure['id'] # 虚假陈述行为id
            self.qinquan_level = eachmeasure['qinquanLevel'] # 侵权等级
            # 揭露日，更正日和发函日中更早的日期
            self.reveal_date = self.correct_date if self.correct_date > self.commit_date and self.correct_date <= self.measure_date else self.measure_date
            
            self.invest_valid = False # 是否符合索赔要求，默认否
            self.bench_mark_date = None
            self.bench_mark_price = None
            self.bench_mark_remark = None
            self.measure_loss = 0 # 名义损失，默认0
            self.measure_sys_loss = 0 # 系统风险损失，默认0
            self.measure_non_sys_loss = 0 # 非系统风险损失，默认0
            self.real_loss = 0 # 实际损失，默认0
            self.other_loss = 0 # 佣金税费利息损失，默认0
            self.net_loss = 0 # 最终损失，默认0
            self.result_remark = ''
        
        # 计算虚假陈述带来的损失
        def cal_measure_loss(self):
            step = '快速校验是否符合索赔条件'
            try:
                # 快速校验是否符合索赔条件
                self.quick_valid(self.slc.stock_trd_hist,self.commit_date,self.reveal_date)
                # 如果揭露日之前无持仓，则中止计算
                if self.init_stock_quant == 0:
                    self.slc.debug('揭露日前无持仓')
                    self.slc.CallBackError('揭露日前无持仓')
                    return 
                
                # 如果通过快速校验，则继续计算
                if self.invest_valid:
                    
                    # 计算基准日期和基准价格
                    step = '计算基准日期和基准价格'
                    self.cal_bench_mark(self.slc.stock_hist,self.reveal_date)
                    # 用实际成本法计算虚假陈述带来的损失
                    step = '用实际成本法计算虚假陈述带来的损失'
                    self.cal_invest_loss_act_cost()
                    
                    # 计算系统性风险带来的损失
                    step = '计算系统性风险带来的损失'
                    self.cal_sys_loss()
                    
                    # 计算非系统性风险带来的损失（虚假陈述以外的其他因素），专业版中新增
                    step = '计算非系统性风险带来的损失（虚假陈述以外的其他因素），专业版中新增'
                    self.cal_non_sys_other_fators_loss()
                    
                    # 计算交易成本损失
                    step = '计算交易成本损失'
                    self.cal_trd_cost_loss()
            except Exception as e:
                self.logger.error(f'{step} error: {e}')
                raise e
        
        # 快速校验是否符合损失要求
        def quick_valid(self,stock_trade_data,commit_date,reveal_date):
    
            invest_valid = False #是否通过快速校验
            invest_init_date = None #首次有效买入日期
            invest_clear_date = None #清仓日期
    
            #截取实施日及之后，揭露日之前的交易数据
            init_stock_trade_data = stock_trade_data.loc[stock_trade_data['d_trade_date']>=commit_date].loc[stock_trade_data['d_trade_date']<reveal_date]
            #获取实施日及之后，揭露日之前，第一笔有效买入日期
            #试算一个第一笔有效买入日期
            #invest_init_date = init_stock_trade_data.loc[stock_trade_data['buy_sell']=='buy']['d_trade_date'].min()
            
            init_cumsum_buy = 0 #揭露日前累计买入
            init_cumsum_sell = 0 #揭露日前累计卖出
            # 逐笔计算，卖出达到或超过买入后，重新计算
            for index,row in init_stock_trade_data.iterrows():
                if row['buy_sell'] == 'buy':
                    if init_cumsum_buy == 0: #第一次买入或超额卖出后的第一次买入
                        invest_init_date = row['d_trade_date']
                    init_cumsum_buy += row['v_trade_quant'] 
                else:
                    init_cumsum_sell += row['v_trade_quant']
                    if init_cumsum_sell >= init_cumsum_buy: #全额超额卖出后，重新计算第一次买入
                        init_cumsum_buy = 0
                        init_cumsum_sell = 0
                        invest_init_date = None
            # 计算揭露日前的持仓
            init_stock_quant = init_cumsum_buy - init_cumsum_sell
            # 如果揭露日前有持仓，则通过快速校验
            invest_valid = init_stock_quant > 0
            
            #截取揭露日及之后的交易数据，找到清仓日期
            clear_stock_trade_data = stock_trade_data.loc[stock_trade_data['d_trade_date']>=reveal_date]
            clear_stock_trade_data['clear_cunsum_sell'] = clear_stock_trade_data.loc[clear_stock_trade_data['buy_sell']=='sell']['v_trade_quant'].cumsum()
            invest_clear_date = clear_stock_trade_data.loc[clear_stock_trade_data['clear_cunsum_sell']>=init_stock_quant]['d_trade_date'].min()
            invest_clear_date = None if not(pd.notna(invest_clear_date)) else invest_clear_date
            
            self.invest_valid = invest_valid
            self.invest_init_date = invest_init_date
            self.init_stock_quant = init_stock_quant
            self.invest_clear_date = invest_clear_date

        # 计算基准日和基准价
        def cal_bench_mark(self,stock_daily,reveal_date):
            
            #基准日分别按下列情况确定
            #（一）揭露日起至被虚假陈述影响的证券累计成交量达到其可流通部分100%之日，但通过大宗交易协议转让的证券成交量不予计算。
            #按第（一）项规定在开庭审理前尚不能确定的，则以揭露日后第30个交易日为基准日。
            #根据《虚假陈述司法解释》第三十二条，基准价=揭露日或者更正日起至基准日期间，每个交易日收盘价的平均价格。

            #找到揭露日之前最近的个股交易日
            pre_reveal_trd_date = stock_daily['date'].loc[stock_daily.loc[stock_daily['date'] < reveal_date].index.max()]
            if pre_reveal_trd_date < stock_daily['date'].loc[stock_daily.index.max()]: #揭露日及之后仍有个股交易
            
                #找到揭露日及之后10个和30个交易日的日期，超过最大可用交易日数据的情况下，取最大交易日（例如，退市、长期停牌等）
                min_bench_mark_date = stock_daily['date'].loc[min(stock_daily.loc[stock_daily['date'] == pre_reveal_trd_date].index.max() + 10,stock_daily.index.max())]
                max_bench_mark_date = stock_daily['date'].loc[min(stock_daily.loc[stock_daily['date'] == pre_reveal_trd_date].index.max() + 30,stock_daily.index.max())]
                
                #找到揭露日之后累计换手率达到100%的日期
                stock_daily_trim = stock_daily.loc[stock_daily['date'] > pre_reveal_trd_date].loc[stock_daily['date'] <= max_bench_mark_date]    
                stock_daily_trim['cumsum_turnover'] = stock_daily_trim['turnover'].cumsum()
                if not(stock_daily_trim.loc[stock_daily_trim['cumsum_turnover']>=100].empty):
                    reach_turnover_date = stock_daily_trim['date'].loc[stock_daily_trim.loc[stock_daily_trim['cumsum_turnover']>=100].index.min()]
                    reach_turnover_in_days = stock_daily_trim.loc[stock_daily_trim['date']==reach_turnover_date].index.min() - stock_daily_trim.index.min() + 1
                    bench_mark_remark = 'cumsum turnover in ' +str(reach_turnover_in_days)+ 'd on ' + str(reach_turnover_date)[0:10]
                else:
                    reach_turnover_date = max_bench_mark_date
                    bench_mark_remark = 'cumsum turnover after 30d'
                
                #比较揭露日后第10、30个交易日与换手率达到100%的日期，确定基准日
                bench_mark_date = min(max(min_bench_mark_date,reach_turnover_date),max_bench_mark_date)
                
                #计算基准价
                bench_mark_price = stock_daily['close'].loc[stock_daily['date']>pre_reveal_trd_date].loc[stock_daily['date']<=bench_mark_date].mean()
            
            else:  #揭露日及之后无个股交易，则取之前最近的一次交易日
                bench_mark_date = pre_reveal_trd_date
            
                #计算基准价
                bench_mark_price = stock_daily.loc[stock_daily['date'] == bench_mark_date]['close'].values[0]
                bench_mark_remark = 'no trades after reveal date'
        
            self.bench_mark_date = bench_mark_date
            self.bench_mark_price = bench_mark_price
            self.bench_mark_remark = bench_mark_remark
    
        # 计算名义投资差额损失
        def cal_invest_loss_act_cost(self):
            
            stock_trade_data = self.slc.stock_trd_hist
            stock_daily = self.slc.stock_hist
            commit_date = self.commit_date
            reveal_date = self.reveal_date
            invest_init_date = self.invest_init_date
            invest_clear_date = self.invest_clear_date
            bench_mark_date = self.bench_mark_date
            bench_mark_price = self.bench_mark_price
            
            #重建index以便计算交易日
            stock_daily = stock_daily.sort_values(by='date').reset_index(drop=True)
        
            #实施日之前的买卖记录不可赔付，trim掉
            stock_trade_data = stock_trade_data.loc[stock_trade_data['d_trade_date']>=commit_date]
            
            #实际成本法
            #根据《虚假陈述司法解释》第三十一条，投资人因证券虚假陈述而受到的损失按如下方式计算
            ##投资人在基准日及以前卖出证券
            ##投资差额损失部分＝（买入证券平均价格-实际卖出证券平均价格）×基准日及以前卖出的股票数量
            ##投资人在基准日之后卖出或者仍持有证券
            ##投资差额损失部分＝（买入证券平均价格-基准价）×基准日后卖出或仍持有的股票数量
            
            #计算实施日及之后，揭露日之前的平均买入价（从实施日后的第一笔买入开始）
            commoit2reveal_stock_trade_data = stock_trade_data.loc[stock_trade_data['d_trade_date']>=invest_init_date].loc[stock_trade_data['d_trade_date']<reveal_date]
            buy_sell_group_sum = commoit2reveal_stock_trade_data.groupby('buy_sell')[['v_trade_quant','v_trade_amount']].sum()    
            if not(commoit2reveal_stock_trade_data.loc[commoit2reveal_stock_trade_data['buy_sell']=='sell'].empty):
                commoit2reveal_init_quant = buy_sell_group_sum.loc['buy','v_trade_quant'] - buy_sell_group_sum.loc['sell','v_trade_quant']
                commoit2reveal_buy_price = (buy_sell_group_sum.loc['buy','v_trade_amount'] - buy_sell_group_sum.loc['sell','v_trade_amount'])/commoit2reveal_init_quant
            else:
                commoit2reveal_init_quant = buy_sell_group_sum.loc['buy','v_trade_quant'] 
                commoit2reveal_buy_price = buy_sell_group_sum.loc['buy','v_trade_amount'] /commoit2reveal_init_quant
        
            #计算揭露日及之后，基准日及之前的平均卖出价，以及该区间的投资差额损失（截止于最早清仓揭露日持仓的日期）
            ##截取数据
            if invest_clear_date == None or invest_clear_date > bench_mark_date:
                reveal2benchmark_stock_trade_data = stock_trade_data.loc[stock_trade_data['d_trade_date']>=reveal_date].loc[stock_trade_data['d_trade_date']<=bench_mark_date]
            else:
                reveal2benchmark_stock_trade_data = stock_trade_data.loc[stock_trade_data['d_trade_date']>=reveal_date].loc[stock_trade_data['d_trade_date']<=invest_clear_date]
            ##进行计算
            if not(reveal2benchmark_stock_trade_data.loc[reveal2benchmark_stock_trade_data['buy_sell']=='sell'].empty):
                buy_sell_group_sum = reveal2benchmark_stock_trade_data.groupby('buy_sell')[['v_trade_quant','v_trade_amount']].sum()    
                reveal2benchmark_sell_quant = buy_sell_group_sum.loc['sell','v_trade_quant']
                reveal2benchmark_sell_price = buy_sell_group_sum.loc['sell','v_trade_amount']/reveal2benchmark_sell_quant
                reveal2benchmark_invest_loss = max((commoit2reveal_buy_price - reveal2benchmark_sell_price) * reveal2benchmark_sell_quant,0)
            else:
                reveal2benchmark_sell_quant = 0
                reveal2benchmark_sell_price = 0
                reveal2benchmark_invest_loss = 0
                
            #计算基准日之后的投资差额损失
            postbenchmark_hold_quant = max(commoit2reveal_init_quant - reveal2benchmark_sell_quant,0) #reveal2benchmark_sell_quant大于commoit2reveal_init_quant时，说明无基准日之后的持仓
            postbenchmark_invest_loss = max((commoit2reveal_buy_price - bench_mark_price) * postbenchmark_hold_quant,0)
            
            #虚假陈述的损失计算结果（名义差额损失，未扣除系统风险损失和非系统风险损失）
            self.measure_loss = int(reveal2benchmark_invest_loss + postbenchmark_invest_loss)
            #如果名义差额损失为0，则提示买入均价低于卖出均价和基准价
            if self.measure_loss == 0:
                self.result_remark = '买入均价低于卖出均价和基准价'
        
        # 计算系统风险带来的损失
        def cal_sys_loss(self):
    
            stock_daily = self.slc.stock_hist
            mkt_index_daily = self.slc.mkt_index_hist
            stock_code = self.stock_code
            invest_init_date = self.invest_init_date
            invest_clear_date = self.invest_clear_date
            bench_mark_date = self.bench_mark_date
            
            #计算系统风险损失
            #资者可获赔投资差额损失=投资差额损失*[1-min（D，0）/G]。
            #上述公式中， D表示选定组合指数在考察区间的平均值，G表示系争股票在考察区间的涨跌幅，
            #如果D＞=0，则表明系争股票未受系统性风险影响；
            #如果D＜0，则表明系争股票受到系统性风险的影响并需要扣除，
            #如果D＜0，且︱D︱＞︱G︱，表明系争股票不仅受到系统性风险影响，而且投资者的损失完全由系统性风险造成。
            #组合指数平均值D=（参考指数1涨跌幅+参考指数2涨跌幅+……参考指数N涨跌幅）/N，N代表参与计算的指数的数量。
            
            #系统风险考察区间起始日期为首次有效买入日期
            cal_sys_loss_st_date = invest_init_date
            #系统风险考察区间结束日期为清仓日期或基准日
            if invest_clear_date == None or invest_clear_date > bench_mark_date:
                cal_sys_loss_ed_date = bench_mark_date
            else:
                cal_sys_loss_ed_date = invest_clear_date
            #找到系统风险考察区间起始日期的股价
            cal_sys_loss_st_date_price_list = stock_daily['close'].loc[stock_daily['date']==cal_sys_loss_st_date]
            if len(cal_sys_loss_st_date_price_list)==0:
                raise Exception(f'未获取到日期：{cal_sys_loss_st_date}, 收盘价，请检查数据')
            invest_st_close_price = cal_sys_loss_st_date_price_list.iloc[0]
            cal_sys_loss_ed_date_price_list = stock_daily['close'].loc[stock_daily['date']==cal_sys_loss_ed_date]
            if len(cal_sys_loss_ed_date_price_list)==0:
                raise Exception(f'未获取到日期：{cal_sys_loss_ed_date},收盘价，请检查数据')
            #找到系统风险考察区间结束日期的股价
            invest_ed_close_price = cal_sys_loss_ed_date_price_list.iloc[0]
            #找到系统风险考察区间的股票损失
            stock_rt = invest_ed_close_price/invest_st_close_price - 1
            
            #假如考察区间股票收益为正，排除系统风险影响，否则需判断股票损失部分多少由系统风险造成
            bench_mark_rt_list = [] #用于存储大盘、行业指数损失
            #系统风险考察区间的股票损失为0，则中止计算，此时系统风险损失为0
            if stock_rt >= 0: 
                measure_sys_loss = 0
                net_loss = self.measure_loss
            else:
            
                #损失计算软件采用了“3+X”的组合参考指标体系。其中，“3”代表综合指数、一级行业指数和三级行业指数（统称“行业指数”），这三类指数是必选指数。
                #“X”代表热点比较突出的反映系争股票特点的概念指数，由同花顺iFinD提供，属于可选指数。
                #参与系统性风险扣除计算的指数必须是按照上述规则判定后最终选取的指数组合，其种类为1至4个。
                try:
                    #计算系统风险考察区间的大盘指数损失
                    mkt_index_rt = mkt_index_daily.loc[mkt_index_daily['date']==cal_sys_loss_ed_date]['close'].iloc[0] \
                        /mkt_index_daily.loc[mkt_index_daily['date']==cal_sys_loss_st_date]['close'].iloc[0] - 1
                except:
                    mkt_index_rt = 0
                try:
                    #计算系统风险考察区间的行业指数损失
                    industry_index_rt = self.cal_ind_index_rt(stock_code,cal_sys_loss_st_date,cal_sys_loss_ed_date)
                except:
                    industry_index_rt = [0,0,0]
                #concept_index_rt = cal_concept_index_rt()  #可选，暂不引入
                #全市场到细分行业依次
                bench_mark_rt_list += [mkt_index_rt]
                bench_mark_rt_list += industry_index_rt
                #bench_mark_rt_list += industry_index_rt
                #综合指数下跌，说明存在全局性系统性风险，4种参考指数均参与计算。
                #综合指数上涨，且一级行业指数下跌，说明存在行业性的系统性风险，故仅一级行业指数、三级行业指数和概念指数参与计算。
                sys_rt_avg = 0
                for eachrt in bench_mark_rt_list:
                    if eachrt > 0:
                        bench_mark_rt_list = bench_mark_rt_list[1:]
                    else:
                        sys_rt_avg = sum(bench_mark_rt_list)/len(bench_mark_rt_list)
                        break
                
                #系统风险损失金额为名义差额损失*综合指数损失占比个股损失
                measure_sys_loss = self.measure_loss * min(min(sys_rt_avg,0)/stock_rt,1)
                net_loss = self.measure_loss - measure_sys_loss
        
            self.cal_sys_loss_st_date = cal_sys_loss_st_date
            self.cal_sys_loss_ed_date = cal_sys_loss_ed_date
            self.measure_sys_loss = int(measure_sys_loss)
            self.net_loss = int(net_loss)
            self.stock_rt = stock_rt
            self.bench_mark_rt_list = bench_mark_rt_list
            
            # 如果系统风险损失为0，则提示个股亏损幅度小于指数亏损幅度
            if self.net_loss == self.measure_loss:
                self.result_remark = '个股亏损幅度小于指数亏损幅度'
        
        # 计算非系统风险损失，暂未实现
        def cal_non_sys_other_fators_loss(self):
            pass
        
        # 计算其他损失，暂未上线
        def cal_trd_cost_loss(self):
        
            pass
            """
            non_sys_invest_loss = self.measure_non_sys_loss
            
            #佣金和印花税损失=投资差额损失部分*印花税比率 + 投资差额损失部分*佣金率*2
            #如客户未提供按印花税千0.5，佣金万8来算
            fare_loss = non_sys_invest_loss * stamp_rate + non_sys_invest_loss * fare_rate * 2
            
            #利息损失=投资差额损失部分*自买入至卖出证券日或者基准日期间*银行同期活期存款利率
            bench_mark_date = bench_mark[0]
            if invest_clear_date == None or invest_clear_date > bench_mark_date:
                interest_loss = non_sys_invest_loss * interest_rate * (bench_mark_date - invest_init_date).days/365
            else:
                interest_loss = non_sys_invest_loss * interest_rate * (invest_clear_date - invest_init_date).days/365
            #投资损失=投资差额损失+佣金和印花税损失+利息损失
            pred_loss = non_sys_invest_loss + fare_loss + interest_loss
            """
        
        # 计算行业指数损失
        def cal_ind_index_rt(self,stock_code,st_date,ed_date):
            
            #计算每层行业指数的收益
            industry_index_rt = []
            for industry_index_id in self.slc.stock_ind_index_layers:
                try:
                    # 获取每个行业指数的历史净值
                    sw_index_hist_daily = self.slc.prepare_sw_ind_index_hist(industry_index_id,st_date,ed_date)
                    # 计算每个行业指数的收益
                    curr_industry_index_rt = sw_index_hist_daily.loc[sw_index_hist_daily['date']==ed_date]['close'].iloc[0] \
                        /sw_index_hist_daily.loc[sw_index_hist_daily['date']==st_date]['close'].iloc[0] - 1
                except:
                    curr_industry_index_rt = 0
                industry_index_rt += [curr_industry_index_rt]
            
            return industry_index_rt

    @staticmethod
    def df2json(df):
        return df.to_json(orient='records')

    @staticmethod
    def json2df(data_json):
        return pd.DataFrame(json.loads(data_json))

    @staticmethod
    def datetime2unixtime(datetime):
        return int(datetime.timestamp())
    
    @staticmethod
    def unixtime2datetime(unixtime):
        return datetime.datetime.fromtimestamp(unixtime, tz=east8) # + relativedelta(hours=8)
    
    #把对象属性转为dataframe
    def class2df(self, class_obj, attributes):
        result = {}
        # 提取对象属性列表
        obj_vars = vars(class_obj)
        for attribute in attributes: # 需要提取的属性名
            if attribute in obj_vars:
                value = obj_vars[attribute]
                if isinstance(value, datetime.datetime):
                    result[attribute] = self.datetime2unixtime(value)
                else:
                    result[attribute] = value
        return result
    
    # 从数据库获取数据
    def fromdb(self,table_name,data_json):
        return self.mxtfapi.execute_api(table_name,'select',data_json)
    
    """
    # 向数据库写入数据，暂时无用
    def add2db(self,table_name,data_json):
        if self.updatedb:
            results = self.mxtfapi.execute_api(table_name,'insert',data_json)
        else:
            results = ''
        return results   
    """

    # 准备和预处理客户交易流水数据
    def prepare_cl_trd_hist(self,cltrdhist_json):
        
        if self.test_mode: #测试模式
            self.calculate_id = self.TaskInfo['caculateId']
            prepare_data_df = self.json2df(cltrdhist_json)
            prepare_data_df['d_trade_time'] = pd.to_datetime(prepare_data_df['trade_date']).dt.tz_localize('Asia/Shanghai')
            prepare_data_df['v_trade_quant'] = pd.to_numeric(prepare_data_df['trade_quant'])
            prepare_data_df['v_trade_amount'] = pd.to_numeric(prepare_data_df['trade_amount'])
        else:
            
            # cltrdhist_json = '[' + cltrdhist_json + ']' if cltrdhist_json[0] != '[' else cltrdhist_json
            # cltrdhist_loaded = json.loads(cltrdhist_json)
            cltrdhist_loaded = [cltrdhist_json]
            self.calculate_id = cltrdhist_loaded[0]['caculateId']
            result = []
            for eachstock in cltrdhist_loaded:
                for eachtrade in eachstock['trade_list']:
                    result.append({
                        'calculateId': eachstock['caculateId'],
                        'stock_code': eachstock['stock_no'],
                        'begin_time': eachstock['begin_time'],
                        'end_time': eachstock['end_time'],
                        
                        'd_trade_time': eachtrade['operationTime'],
                        'buy_sell': str(eachtrade['direction']).replace('0', 'sell').replace('1', 'buy'),
                        'v_trade_price': eachtrade['price'],
                        'v_trade_quant': eachtrade['qty'],
                        'v_trade_amount': eachtrade['amount']
                    })
                    
            prepare_data_df = pd.DataFrame(result)
            datecols = ['d_trade_time']
            for eachcol in datecols:
                prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        
        # 排序并转换交易日期时间格式
        prepare_data_df = prepare_data_df.sort_values(['stock_code','d_trade_time'])    
        prepare_data_df['d_trade_date'] = prepare_data_df['d_trade_time'].dt.normalize()
        
        return prepare_data_df

    # 准备和预处理复权因子数据
    def prepare_stock_price_adj_factor(self,symbol,st_date,ed_date,factor_type='qfq'):
        
        # 获取数据api的参数
        end_date = ed_date.strftime('%Y-%m-%d') + ' 23:59:59'
        start_date = st_date.strftime('%Y-%m-%d') + ' 00:00:00'
        data_json = json.dumps({
            "stock_code":symbol,
            "start_date":start_date,
            "end_date":end_date,
            "factor_type":factor_type
            })
        table_name = 'stock_price_adj_factor'
        # 调用api获取数据
        stock_price_adj_factor = self.fromdb(table_name,data_json)
        if stock_price_adj_factor=='[]':
            return False
        prepare_data_df = self.json2df(stock_price_adj_factor)
        # 转换日期字段格式
        datecols = ['createdAt','updatedAt','date']
        for eachcol in datecols:
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        self.stock_price_adj_factor = prepare_data_df
        if prepare_data_df.empty:
            return False
        return True

    # 计算复权后的价格
    def calculate_fuquan_price(self,stock_code,price_hist,factor_type='qfq'):
        
        st_date = price_hist['date'].min()
        ed_date = price_hist['date'].max()
        
        # 转换股票代码格式
        if stock_code[:3] in self.stockcode2mkt:
            symbol = self.stockcode2mkt[stock_code[:3]] + stock_code
        else:
            symbol = 'sh' + stock_code
        
        # 准备复权因子
        factor_exist = self.prepare_stock_price_adj_factor(symbol,st_date,ed_date,factor_type)
        
        if factor_exist:
            # 寻找小于或等于price_hist.date 的最大stock_price_adj_factor.date
            original_index = price_hist.index
            price_hist = pd.merge_asof(price_hist,self.stock_price_adj_factor,on='date',direction='backward')
            price_hist = price_hist.set_index(original_index)
            # 计算复权价格
            price_hist['value'] = price_hist['value'] /price_hist['factorValue'] * 10**12
        
        return price_hist['value']

    # 准备和预处理个股交易流水
    def prepare_stock_trd_hist(self,stock_code):
        
        # 从客户交易流水截取个股交易流水
        stock_trd_hist = self.cl_trd_hist[self.cl_trd_hist['stock_code']==stock_code]
        # 转换字段名称，预处理数据格式
        price_hist = stock_trd_hist[['d_trade_date','v_trade_amount']]
        price_hist.columns = ['date','value']
        stock_trd_hist['v_trade_amount'] = self.calculate_fuquan_price(stock_code,price_hist,'qfq')
        stock_trd_hist['v_trade_price'] = stock_trd_hist['v_trade_amount']/stock_trd_hist['v_trade_quant']
        
        self.stock_trd_hist = stock_trd_hist
    
    """
    # 准备和预处理虚假陈述行为数据，暂未使用
    def prepare_stock_measure(self,stock_code,horizon=10):
        
        # 获取数据api的参数
        now = datetime.datetime.now()
        end_date = now.strftime('%Y-%m-%d') + ' 23:59:59'
        start_date = (now - relativedelta(years=horizon)).strftime('%Y-%m-%d') + ' 00:00:00'
        data_json = json.dumps({
            "stock_code":stock_code,
            "start_date":start_date,
            "end_date":end_date
            })
        table_name = 'exchange_measure_hist'
        measure_hist = self.fromdb(table_name,data_json)
        if measure_hist=='[]':
            return False
        # 转换日期字段格式
        prepare_data_df = self.json2df(measure_hist)
        prepare_data_df = prepare_data_df[prepare_data_df['judgeResult']=='是']
        datecols = ['createdAt','updatedAt','measureDate','xujiachenshuDate','gengzhengDate']
        for eachcol in datecols:
            print(eachcol)
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        self.xujiachenshu_action = prepare_data_df
        if prepare_data_df.empty:
            return False
        return True
    """
    
    # 准备和预处理虚假陈述行为数据
    def prepare_xujiachenshu_action(self,stock_code,horizon=10):
        
        # 获取数据api的参数
        now = datetime.datetime.now()
        end_date = now.strftime('%Y-%m-%d') + ' 23:59:59'
        start_date = (now - relativedelta(years=horizon)).strftime('%Y-%m-%d') + ' 00:00:00'
        data_json = json.dumps({
            "stock_code":stock_code,
            "start_date":start_date,
            "end_date":end_date
            })
        table_name = 'xujiachenshu_action'
        xujiachenshu_action = self.fromdb(table_name,data_json)
        if xujiachenshu_action=='[]':
            return False
        # 转换日期字段格式
        prepare_data_df = self.json2df(xujiachenshu_action)
        prepare_data_df = prepare_data_df[prepare_data_df['xujiachenshuDate']!=0]
        if prepare_data_df.empty:
            return False
        datecols = ['createdAt','updatedAt','measureDate','xujiachenshuDate','gengzhengDate']
        for eachcol in datecols:
            print(eachcol)
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        self.xujiachenshu_action = prepare_data_df
        if prepare_data_df.empty:
            return False
        return True

    # 准备和预处理股票行情数据
    def prepare_stock_hist(self,stock_code,horizon=10):
        
        # 获取数据api的参数
        now = datetime.datetime.now()
        end_date = now.strftime('%Y-%m-%d') + ' 23:59:59' 
        try:
            start_date = pd.to_datetime(self.xujiachenshu_action['xujiachenshuDate']).min().strftime('%Y-%m-%d') + ' 00:00:00' 
        except:
            start_date = (now - relativedelta(years=horizon)).strftime('%Y-%m-%d') + ' 00:00:00'
        data_json = json.dumps({
            "stock_id":stock_code,
            "start_date":start_date,
            "end_date":end_date
            })
        table_name = 'stock_hist'
        datas=None
        for i in range(2):
            try:
                datas=self.fromdb(table_name,data_json)
                break
            except Exception as e:
                self.debug(e)
        if not datas:
            raise Exception('获取历史股价出错')
        # 转换日期字段格式
        prepare_data_df = self.json2df(datas)    
        datecols = ['createdAt','updatedAt','date']
        for eachcol in datecols:
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        stock_hist = prepare_data_df.sort_values(by='date').reset_index(drop=True)

        price_hist = stock_hist[['date','amount']]
        price_hist.columns = ['date','value']
        stock_hist['amount'] = self.calculate_fuquan_price(stock_code,price_hist,'qfq')        

        self.stock_hist = stock_hist

    # 准备和预处理大盘指数行情数据
    def prepare_mkt_index_hist(self,stock_code,horizon=10):
        
        # 获取数据api的参数
        if stock_code[:3] in self.stockcode2mkt:
            symbol = self.mkt2mktindex[self.stockcode2mkt[stock_code[:3]]]
        else:
            symbol = 'sh000001'
        now = datetime.datetime.now()
        end_date = now.strftime('%Y-%m-%d') + ' 23:59:59' 
        try:
            start_date = pd.to_datetime(self.xujiachenshu_action['xujiachenshuDate']).min().strftime('%Y-%m-%d') + ' 00:00:00' 
        except:
            start_date = (now - relativedelta(years=horizon)).strftime('%Y-%m-%d') + ' 00:00:00' 
        data_json = json.dumps({
            "symbol":symbol,
            "start_date":start_date,
            "end_date":end_date
            })
        table_name = 'mkt_index_hist'
        prepare_data_df = self.json2df(self.fromdb(table_name,data_json))   
        # 转换日期字段格式
        datecols = ['createdAt','updatedAt','date']
        for eachcol in datecols:
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        self.mkt_index_hist = prepare_data_df

    # 准备和预处理申万行业成分变更数据
    def prepare_sw_ind_comp_change_hist(self,stock_code):
        
        data_json = json.dumps({
            "symbol":stock_code
            })
        table_name = 'sw_ind_comp_change_hist'
        prepare_data_df = self.json2df(self.fromdb(table_name,data_json))    
        datecols = ['createdAt','updatedAt','startDate','updateTime']
        for eachcol in datecols:
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        self.sw_ind_comp_change_hist = prepare_data_df
        return prepare_data_df

    # 准备和预处理申万行业代码和指数对应关系
    def prepare_sw_ind_index_dict(self):

        data_json = ''
        table_name = 'sw_ind_index_dict'
        prepare_data_df = self.json2df(self.fromdb(table_name,data_json))    
        datecols = []
        for eachcol in datecols:
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        self.sw_ind_index_dict = prepare_data_df
        return prepare_data_df

    # 准备个股所在申万行业分类信息
    def prepare_sw_ind_index_layers(self,stock_code):
        
        # 准备申万行业数据
        sw_hist_comp = self.prepare_sw_ind_comp_change_hist(stock_code)
        sw_industry_info = self.prepare_sw_ind_index_dict()
        
        # 找到股票对应的行业指数代码
        stock_industry_id = sw_hist_comp['industryCode'].iloc[0]
        stock_industry_info = sw_industry_info.loc[sw_industry_info['industryId']==stock_industry_id].iloc[0]
        stock_l1_industry_index_id = sw_industry_info.loc[sw_industry_info['industryIndexType']=='一级行业'].loc[sw_industry_info['firstIndustry']==stock_industry_info['firstIndustry']]['industryIndexId'].iloc[0]
        stock_l2_industry_index_id = sw_industry_info.loc[sw_industry_info['industryIndexType']=='二级行业'].loc[sw_industry_info['secondIndustry']==stock_industry_info['secondIndustry']]['industryIndexId'].iloc[0]
        stock_l3_industry_index_id = sw_industry_info.loc[sw_industry_info['industryIndexType']=='三级行业'].loc[sw_industry_info['thirdIndustry']==stock_industry_info['thirdIndustry']]['industryIndexId'].iloc[0]
        stock_ind_index_layers = [stock_l1_industry_index_id,stock_l2_industry_index_id,stock_l3_industry_index_id]

        self.stock_ind_index_layers = stock_ind_index_layers

    # 准备和预处理申万行业指数行情数据
    def prepare_sw_ind_index_hist(self,index_id,start_date,end_date):
        
        start_date = start_date.strftime('%Y-%m-%d') + ' 00:00:00'
        end_date = end_date.strftime('%Y-%m-%d') + ' 23:59:59'
        data_json = json.dumps({
            "swIndIndexId":index_id,
            "start_date":start_date,
            "end_date":end_date
            })
        table_name = 'sw_ind_index_hist'
        prepare_data_df = self.json2df(self.fromdb(table_name,data_json))    
        datecols = ['createdAt','updatedAt','date']
        for eachcol in datecols:
            prepare_data_df[eachcol] = prepare_data_df[eachcol].apply(lambda x: self.unixtime2datetime(x))
        self.sw_ind_index_hist = prepare_data_df
        return prepare_data_df
    def debug(self,msg):
        self.logger.debug(f"caculateId:{self.TaskInfo.get('caculateId')} {msg}")
    
    # 主计算代码
    def Caculate(self,callback=False):
        try:
            
            # 对客户交易历史中的每个股票进行索赔损失计算
            for eachstock in self.stock_list:
                
                """
                # 把股票代码转为股票ID，eg. 600123 -> sh600123
                if eachstock[:3] in self.stockcode2mkt:
                    eachstockid = self.stockcode2mkt[eachstock[:3]] + eachstock
                else:
                    eachstockid = 'sh' + eachstock
                """
                self.debug("准备损失计算所需数据")
                # 准备损失计算所需数据
                try: 
                    # 准备股票交易数据
                    self.prepare_stock_trd_hist(eachstock)
                except:
                    self.debug("拆解交易流水出错")
                    return
                self.debug("获取监管数据")
                # 准备虚假陈述行为数据
                # havaHis = self.prepare_stock_measure(eachstock)
                havaHis = self.prepare_xujiachenshu_action(eachstock)
                if havaHis == False:
                    self.debug("不存在监管措施记录")
                    self.CallBackError('不存在监管措施记录')
                    return
                self.debug(f"获取股票历史股价{self.TaskInfo.get('stock_no')}")
                # 准备股票行情数据
                self.prepare_stock_hist(eachstock)
                
                # 检查股票成交流水
                stock_trd_hist_dates = self.stock_trd_hist[self.stock_trd_hist['d_trade_date']>=self.xujiachenshu_action['xujiachenshuDate'].min()]['d_trade_date'].drop_duplicates().unique() 
                stock_hist_dates = self.stock_hist['date'].drop_duplicates().unique() 
                # 检查股票成交流水是否发生在最早行情记录之前，若是，则中止计算
                stock_trd_last_date = self.stock_trd_hist[self.stock_trd_hist['d_trade_date']>=self.xujiachenshu_action['xujiachenshuDate'].min()]['d_trade_date'].max()
                stock_hist_first_date = self.stock_hist['date'].min()
                if stock_hist_first_date > stock_trd_last_date:
                    self.debug(f"交易时间过早，需包含{stock_hist_first_date:%Y-%m-%d}以后的数据")
                    self.CallBackError(f"交易时间过早，需包含{stock_hist_first_date:%Y-%m-%d}以后的数据")
                    return
                # 检查股票成交流水是否发生在非开盘日，若是，则中止计算
                diff_dates = list(set(stock_trd_hist_dates) - set(stock_hist_dates))
                err_diff_dates = [diffdate.strftime("%Y%m%d") for diffdate in diff_dates[:3]]
                if len(diff_dates) >0:
                    self.debug(f"交易流水错误，{err_diff_dates}等日期未开盘")
                    self.CallBackError(f"交易流水错误，{err_diff_dates}等日期未开盘")
                    return
                
                self.debug("获取大盘历史")
                self.prepare_mkt_index_hist(eachstock)
                self.debug("获取申万行业")
                self.prepare_sw_ind_index_layers(eachstock)
                
                # 创建一个对象后续用来存损失最大的那次计算
                self.opt_mlc = None
                # 对每个股票的每次监管措施都计算损失
                n = 0
                for _index , eachmeasure in self.xujiachenshu_action.iterrows():
                    try:
                        n+=1
                        #if n ==4:
                        #    break
                        # 计算每次监管措施的损失
                        self.debug("计算每次监管措施的损失")
                        self.new_mlc = self.measure_loss_calculator(self,eachmeasure)
                        self.new_mlc.cal_measure_loss()
                        self.summary['more_details'][(eachstock,n)] = self.class2df(self.new_mlc,self.key_attr)
                        self.debug("找到损失最大的那次计算实例")
                        # 找到损失最大的那次计算实例
                        if self.opt_mlc:
                            if self.new_mlc.net_loss > self.opt_mlc.net_loss:
                                self.opt_mlc = self.new_mlc
                        else:
                            self.opt_mlc = self.new_mlc
                    except Exception as e:
                        self.logger.error(f'计算:{eachmeasure.to_json()} 失败 caculate error:{e}')
                        raise e
                    
                self.debug("汇总所有个股损失计算结果")
                # 汇总所有个股损失计算结果
                if self.opt_mlc:
                    self.summary['total_loss'] += self.opt_mlc.net_loss
                    self.summary['details'][eachstock] = self.class2df(self.opt_mlc,self.key_attr)
                else:
                    self.summary['details'][eachstock] = {}
            if callback:
                self.debug("回调")
                self.CallBack()
        except Exception as e:
            self.logger.error(f'caculate error:{e}')
            self.CallBackError(e)
    def CallBackError(self,error):
        self.callbackdata = {}
        self.callbackdata['status'] = 4
        self.callbackdata['caculateId'] = self.TaskInfo['caculateId']
        self.callbackdata['ammount'] = 0
        self.callbackdata['measureType'] = ''
        self.callbackdata['measureSubtype'] = ''
        self.callbackdata['beginDate'] = self.TaskInfo['begin_time']
        self.callbackdata['endDate'] = self.TaskInfo['end_time']
        self.callbackdata['error'] = str(error)
        self.debug(f"回调数据:{self.callbackdata}")
        self.mxtfapi.CallBackCaulculateResult(self.callbackdata)
    def CallBack(self,):
        
        results = self.summary
        # 目前只会计算一只股票，回写第一只即可
        callbackkey = list(results['details'].keys())[0]
        results_slice = results['details'][callbackkey]
        
        self.callbackdata = {}
        self.callbackdata['status'] = 10
        self.callbackdata['caculateId'] = results_slice['calculate_id']
        self.callbackdata['ammount'] = results_slice['net_loss']
        self.callbackdata['measureType'] = results_slice['measure_type']
        self.callbackdata['measureSubtype'] = results_slice['measure_subtype']
        self.callbackdata['beginDate'] = results_slice.get('cal_sys_loss_st_date',0)
        self.callbackdata['endDate'] = results_slice.get('cal_sys_loss_ed_date',0)
        self.callbackdata['error'] = results_slice.get('result_remark',0)
        self.debug(f"回调数据:{self.callbackdata}")
        self.mxtfapi.CallBackCaulculateResult(self.callbackdata)
        
        self.processdata = {}
        self.processdata['lossId'] = results_slice['calculate_id']
        self.processdata['investInitDate'] = results_slice['invest_init_date']
        self.processdata['initStockQuant'] = results_slice['init_stock_quant']
        self.processdata['violationId'] = results_slice['action_id']
        self.processdata['qinquanLevel'] = results_slice['qinquan_level']
        self.processdata['commitDate'] = results_slice['commit_date']
        self.processdata['revealDate'] = results_slice['reveal_date']
        self.processdata['benchMarkDate'] = results_slice['bench_mark_date']
        self.processdata['benchMarkPrice'] = results_slice['bench_mark_price']
        self.processdata['nominalLoss'] = results_slice['measure_loss']
        try:
            self.processdata['sysLossRatio'] = results_slice['measure_sys_loss']/results_slice['measure_loss']
        except:
            self.processdata['sysLossRatio'] = 0
        try:
            self.processdata['nonSysLossRatio'] = results_slice['measure_non_sys_loss']/results_slice['measure_loss']
        except:
            self.processdata['nonSysLossRatio'] = 0
        self.processdata['realLoss'] = results_slice['real_loss'] 
        self.processdata['otherLoss'] = results_slice['other_loss']
        self.processdata['totalLoss'] = results_slice['net_loss']
        self.debug(f"回调过程数据:{self.processdata}")
        self.mxtfapi.addCaculateProcess(self.processdata)
        
    
if __name__ == '__main__':
    
    if 1==1:
        # 测试用客户交易数据
        testmode = True
        callback = False
        testfilepath = r'C:\Users\Julin\Downloads\guan_calculator_test_cl_trd_hist.xlsx'
        testdata = pd.read_excel(testfilepath,dtype=str)
        cl_trd_hist_json = testdata.to_json(orient='records')

    else:
        testmode = False
        callback = True
        cl_trd_hist_json = sys.argv[1]
    
    lc = LossCalculator(cl_trd_hist_json,testmode)
    lc.Caculate(callback)
    results = lc.summary
    lc.CallBack()
    
    